import requests
from lxml import etree

# 请求头
headers = {
    "user-agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/127.0.0.0 Safari/537.36"
}

with open("../../data/movies.txt", mode="w", encoding="utf-8") as f:
    with open("../../data/comments.txt", mode="w", encoding="utf-8") as c_f:
        # 循环爬取所有页
        for page in range(1, 3):
            # 构建开始位置
            start = 25 * (page - 1)
            # 发起请求
            response = requests.get(url=f"https://movie.douban.com/top250?start={start}&filter=", headers=headers)
            # 转换成html对象
            html = etree.HTML(response.text)
            # 循环解析每一个电影
            for item in html.xpath("//div[@class='item']"):
                name = item.xpath("div/div/a/span/text()")[0]

                print(f"正在爬取：{name}")

                # 获取电影详细信息的网址
                url = item.xpath("div/div/a/@href")[0]

                # 获取电影编号
                id = url.split("/")[-2]

                # 获取电影详细信息
                info_response = requests.get(url=url, headers=headers)

                info_html = etree.HTML(info_response.text)

                # 解析网页

                # 导演
                directer = "|".join(info_html.xpath("//div[@id='info']/span/span/a[@rel='v:directedBy']/text()"))

                # 编剧
                screenwriter = "|".join(info_html.xpath("//div[@id='info']/span[2]/span/a/text()"))

                # 主演
                starring = "|".join(info_html.xpath("//div[@id='info']/span/span/a[@rel='v:starring']/text()"))

                # 电影类型
                genre = "|".join(info_html.xpath("//div[@id='info']/span[@property='v:genre']/text()"))

                # 获取div中所有的文本
                div = info_html.xpath("//div[@id='info']/text()")

                # 过滤无效数据
                div = list(filter(lambda i: len(i.strip().replace("/", "")) != 0, div))

                if len(div) == 0:
                    continue

                # 国家地区
                city = div[0].replace(" ", "").replace("/", "|")

                # 语言
                language = div[1].replace(" ", "").replace("/", "|")

                # 上映时间
                release_date = "|".join(
                    info_html.xpath("//div[@id='info']/span[@property='v:initialReleaseDate']/text()"))

                # 片长
                runtime = "|".join(info_html.xpath("//div[@id='info']/span[@property='v:runtime']/text()"))

                # 评分
                score = "|".join(info_html.xpath("//strong[@property='v:average']/text()"))

                # 描述
                desc = "".join(info_html.xpath("//span[@property='v:summary']/text()")) \
                    .replace("\n", "") \
                    .replace(" ", "") \
                    .replace("\t", "") \
                    .replace("　", "")

                # 将数据保存到文件中
                data = f"{id}\t{name}\t{url}\t{directer}\t{screenwriter}\t{starring}\t{genre}\t{city}\t{language}\t{release_date}\t{runtime}\t{score}\t{desc}\n"

                f.write(data)

                # 爬取电影评价数据
                for comment_page in range(1, 11):

                    # 睡眠一会，避免太频繁，ip被封
                    import time

                    # time.sleep(0.1)

                    comment_start = 20 * (comment_page - 1)
                    # 构建电影评价网址
                    comments_url = f"https://movie.douban.com/subject/{id}/comments?start={comment_start}&limit=20&status=P&sort=new_score"
                    # 发起请求获取平均内容
                    comments_response = requests.get(comments_url, headers=headers)
                    # 转换成html对象
                    comments_html = etree.HTML(comments_response.text)

                    # 循环解析每一个评价
                    for comment in comments_html.xpath("//div[@class='comment']"):
                        # 评价内容
                        comment_content = "|".join(comment.xpath("p/span/text()")) \
                            .replace("\n", "") \
                            .replace("\r", "") \
                            .replace(" ", "") \
                            .replace("\t", "") \
                            .replace("　", "")

                        # 评分
                        comment_score = comment.xpath("h3/span/span[2]/@class")[0][7:8]

                        # 评价时间
                        comment_time = comment.xpath("h3/span/span[@class='comment-time ']/text()")[0].strip()

                        # 评价用户
                        user = comment.xpath("h3/span[@class='comment-info']/a/text()")[0]

                        # 保存到文件
                        c_f.write(f"{id}\t{name}\t{comment_score}\t{comment_time}\t{user}\t{comment_content}\n")
